package com.spider.ask120.webmagic;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jdiy.json.JsonArray;
import org.jdiy.json.JsonObject;
import org.jdiy.json.JsonUtil;

import com.spider.utils.HttpClientUtil;
import com.wilddog.client.DataSnapshot;
import com.wilddog.client.Query;
import com.wilddog.client.ValueEventListener;
import com.wilddog.client.Wilddog;
import com.wilddog.client.WilddogError;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 健康经验正文
 * @author Jxw
 *
 */
public class JingyanBAK  implements PageProcessor{
	static String jkjyUrl = "http://m.120ask.com/jingyan/";
	static String listUrl = "http://m.120ask.com/jingyan/list/\\w+"; 
	
	Wilddog dog = new Wilddog("https://cyqa.wilddogio.com/article/jingyan");
	
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);
	public Site getSite() {  
		return site;
	}
 
	public void process(Page page) { 
		
		//进入频道页,获取各病种列表页URL
		if(page.getUrl().regex(jkjyUrl+"$").match()) { 
			//获取到话题页列表URL 
			List<String> huati = page.getHtml().links().regex(listUrl).all(); 
			page.addTargetRequests(huati);
		 
		}else if(page.getUrl().regex(listUrl).match()) {//不包括.html后缀  //+"$" 
			
			String path = page.getUrl().toString();
			//获取?cat= 后面的目录
			String[] cats = (path.indexOf("?cat=")>0 ? path.substring(path.indexOf("?cat=")+5) : "").split(",");
			
			
			//无下级栏目，开始加载分页数据列表，获取正文URL
			Map<String, String> params = new HashMap<String, String>();
			params.put("ajax", "1");
			int pageIndex = 1;	
			while(true){ 
				//从第一页开始取， 只能拿到HTML页面内容，取不到json格式的数据，所以跳过第一页；
				pageIndex ++;
				params.put("page", pageIndex+"");
				String listJson = HttpClientUtil.getInstance().sendHttpPost(page.getUrl().toString(), params);
				if(listJson != null && !listJson.isEmpty()){  
					try {
						JsonObject rstObj = (JsonObject) JsonUtil.parse(listJson); 
						//{"code":200,"list":[]}
						if(rstObj != null && rstObj.get("code") != null && "200".equals(rstObj.get("code").toString()) 
										&& rstObj.get("list") != null && !"[]".equals(rstObj.get("list").toString())){
							
							JsonArray objAry = (JsonArray) JsonUtil.parse(rstObj.get("list").toString());
							if(objAry.size() > 0){ 
								for(int i = 0; i < objAry.size(); i++){
									final JsonObject obj = (JsonObject) objAry.get(i); 

									Map<String, Object> map = new HashMap<>();
									map.put("id", obj.get("id"));
									map.put("id_str", obj.get("id_str"));
									map.put("title", obj.get("title"));
									map.put("content", obj.get("content"));
									map.put("pic", obj.get("pic"));
									
									if(cats.length > 0){ 
										String childPath = "";
										Map<String, Object> catmap = new HashMap<>();
										for(int c = 0; c < cats.length; c++){
											catmap.put(cats[c], true);
											if(c == cats.length - 1){
												childPath += "/"+cats[c];
											}else{
												childPath += "/"+cats[c]+"/child"; 
											}
										}
										map.put("catalog", catmap);
										//将文章ID放入对应的栏目下
										Wilddog channel = new Wilddog("https://cyqa.wilddogio.com/channel/jingyan/child"+childPath);
										System.err.println("https://cyqa.wilddogio.com/channel/jingyan/child"+childPath);
										final Query query = channel.orderByKey();
										query.addListenerForSingleValueEvent(new ValueEventListener() { 
											public void onDataChange(DataSnapshot shot) {
												if(shot.exists()){
													Wilddog article = query.getRef().child("article");
													Map<String, Object> article_id = new HashMap<String, Object>();
													article_id.put(obj.get("id").toString(), true);
													article.updateChildren(article_id);
												}
											} 
											public void onCancelled(WilddogError err) {
												System.err.println(err.getMessage() +" ----- "+err.getDetails());
											}
										});
									}
									
//									Wilddog ref = dog.child("basic/"+obj.get("id").toString());
//									ref.setValue(map);
									
									//进入到文章正文
//									Map<String, Object> extras = new HashMap<String, Object>();
//									extras.put("article_id", obj.get("id")); 
//									Request request = new Request("http://m.120ask.com/jingyan/"+obj.get("id_str")+".html");
//									request.setExtras(extras);
//									page.addTargetRequest(request);
								}
							}else{ 
								break;
							}
						}else{ 
							break;
						}
					} catch (Exception e) { 
						e.printStackTrace();
					}
				}
				
				//只取一页的数据
				break;
			}
			
			 
		}else if(page.getUrl().regex(jkjyUrl+"\\w+.html").match()) {
			String title = page.getHtml().xpath("//div[@class='g-jinyan-topic']/h3/text()").get();
			String create_time = page.getHtml().xpath("//div[@class='g-jinyan-topic']/p[1]/text()").regex("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}").get();
			String share = page.getHtml().xpath("//div[@class='g-jinyan-topic']/p[1]/text()").toString();
			share = share.substring(share.indexOf("由")+1, share.indexOf("分享")).trim(); 
			String abst = page.getHtml().xpath("//div[@class='g-jinyan-cont']/p/text()").get();			//介绍
			String remind = page.getHtml().xpath("//div[@class='g-jinyan-conf']/p[2]/html()").get();	//注意事项
			List<String> pageContent = page.getHtml().xpath("//div[@class='d1 img']/dl/dd/html()").all();		//步骤方法
			List<String> keyword = page.getHtml().xpath("//div[@class='g-jinyan-topic']/p[2]/a/text()").all();
			
			
			Map<String, Object> map = new HashMap<>();
			map.put("title", title);
			map.put("create_time", create_time);
			map.put("share", share);
			map.put("abst", abst);
			map.put("remind", remind);
			Map<String, Object> contentMap = new HashMap<>();
			for(int i = 0; i < pageContent.size(); i++){
				contentMap.put(i+"", pageContent.get(i));  
			}
			map.put("content", contentMap);

			Map<String, Object> keywordMap = new HashMap<>();
			for(int i = 0; i < keyword.size(); i++){
				keywordMap.put(i+"", keyword.get(i));  
			}
			map.put("keyword", keywordMap);
			
			String article_id = page.getRequest().getExtra("article_id").toString();
			Wilddog ref = dog.child("detail/"+article_id);
			ref.setValue(map);
			
		}
		
		
		

	}
  
	public static void main(String[] args) {
		Spider.create(new JingyanBAK()).addUrl("http://m.120ask.com/jingyan/list/aowmao3q3l").thread(1).run(); 
    }
 
}
